# -*- coding: utf-8 -*-
import scrapy
from scrapy.linkextractors import LinkExtractor
from scrapy.spiders import CrawlSpider, Rule
from ..items import PoemItem,AuthorItem
import re
from lxml import etree
import time,html


class JsSpider(CrawlSpider):
    name = 'js'
    allowed_domains = ['gushiwen.org']
    start_urls = ['https://so.gushiwen.org/authors/default.aspx?p=1&c=']

    rules = (
        Rule(LinkExtractor(allow=r'/authors/default.aspx\?p=\d&c='), follow=True,callback='se'),
        Rule(LinkExtractor(allow=r'/authors/authorvsw_[0-9a-z]{12}A\d.aspx'), follow=True,callback='parse_item'),
    )

    def se(self,response):
        html=response.body
        html=html.decode()
        html=etree.HTML(html)
        time.sleep(2)
        authors=html.xpath('//div[@class="left"]/div[@class="sonspic"]')
        for auth in authors:
            author=auth.xpath('./div[@class="cont"]/p[1]//b/text()')[0]
            simple=auth.xpath('./div[@class="cont"]/p[2]/text()')[0]
            img=auth.xpath('./div[@class="cont"]/div[@class="divimg"]//img/@src')[0]
            item=AuthorItem(author=author,simple=simple,img=img)
            print(item)
            yield item


    def parse_item(self, response):
        html1 = response.body
        html1 = html1.decode()
        html1 = etree.HTML(html1)
        # time.sleep(2)
        poems=html1.xpath('//div[@class="left"]/div[@class="sons"]')
        print(poems[0])
        for poem in poems:
            body=poem.xpath('./div[@class="cont"]/div[@class="contson"]')[0]
            body = html.unescape(str(etree.tostring(body)))#把那些该死的#$开头的东西装换成中文
            body = re.sub('<.*?>', '', body)
            body = re.sub('[b\']', '', body)
            body = re.sub(r'\\n', '', body)#把弄出来的诗句，把换行和标签都给去了
            title=poem.xpath('./div[@class="cont"]/p[1]//b/text()')[0]
            dynasty=poem.xpath('./div[@class="cont"]/p[2]/a[1]/text()')[0]
            auth=poem.xpath('./div[@class="cont"]/p[2]/a[2]/text()')[0]
            the_tags=poem.xpath('./div[@class="tag"]/a')
            tags=[] #这是这首诗句的标签
            for tag in the_tags:
                tags.append(tag.xpath('./text()')[0])
            item=PoemItem(body=body,title=title,tags=tags,author=auth,dynasty=dynasty)
            print(item)
            yield item




